{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Verify the performance of the model in the validation set\n",
    "the performance of our models in the validation set\n",
    "\n",
    "|model name | ACER|TPR@FPR=10E-2|TPR@FPR=10E-3|FP|FN|epoch|params|FLOPs|\n",
    "| ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ |\n",
    "|FishNet150| 0.00144|0.999668|0.998330|19|0|27|24.96M|6452.72M|\n",
    "|FishNet150| 0.00181|1.0|0.9996|24|0|52|24.96M|6452.72M|\n",
    "|FishNet150| 0.00496|0.998664|0.990648|48|8|16|24.96M|6452.72M|\n",
    "|MobileNet v2|0.00228|0.9996|0.9993|28|1|5|2.23M|306.17M\n",
    "|MobileNet v2|0.00387|0.999433|0.997662|49|1|6|2.23M|306.17M\n",
    "|MobileNet v2|0.00402|0.9996|0.992623|51|1|7|2.23M|306.17M\n",
    "|FeatherNet54|0.00242|1.0|0.99846|32|0|41|0.57M|270.91M|\n",
    "|FeatherNet54-se|0.00242|1.0|0.996994|32|0|69|0.57M|270.91M|\n",
    "|MobileLiteNetA|0.00261|1.00|0.961590|19|7|51|0.35M|79.99M|\n",
    "|MobileLiteNetB|0.00168|1.0|0.997662|20|1|48|0.35M|83.05M|\n",
    "|**Ensembled all**|0.0000|1.0|1.0|0|0|-|-|-|\n",
    "\n",
    "Run the following command, you can see the performance of the model in the test set in the ./submission/ directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume checkpoints/fishnet150_bs32/_15.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume checkpoints/fishnet150_bs32/_51_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/fishnet150-32.yaml\" --resume checkpoints/fishnet150_bs32/_26_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/FeatherNet54-32.yaml\" --resume checkpoints/FeatherNet54/_40_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/FeatherNet54-se-64.yaml\" --resume checkpoints/FeatherNet54-se/_68_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume checkpoints/mobilenetv2_bs32/_4_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume checkpoints/mobilenetv2_bs32/_5.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/mobilenetv2.yaml\" --resume ./checkpoints/mobilenetv2_bs32/_6.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/MobileLiteNetA-32.yaml\" --resume ./checkpoints/mobilelitenetA_bs32/_50_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!python main.py --config=\"cfgs/MobileLiteNetB-32.yaml\" --resume ./checkpoints/mobilelitenetB_bs32/_47_best.pth.tar --val True --val-save True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## split the predicted scores from each submission file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def splitscore(file_dir):\n",
    "    score = []\n",
    "    Prefix_str = []\n",
    "    f = open(file_dir)\n",
    "    for line in f:\n",
    "        s =line.split()\n",
    "        score.append(float(s[-1]))\n",
    "        s = s[0] + ' ' + s[1] + ' ' + s[2] + ' '\n",
    "        Prefix_str.append(s)\n",
    "    return score,Prefix_str"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect all predicted scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.238889e-05\n"
     ]
    }
   ],
   "source": [
    "file_dir1='submission/2019-01-28_15:45:05_fishnet150_52_submission.txt'\n",
    "score1,Prefix_str = splitscore(file_dir1)\n",
    "file_dir2 = 'submission/2019-02-13_15:22:05_FeatherNet54-se_69_submission.txt'\n",
    "score2,Prefix_str = splitscore(file_dir2)\n",
    "# print(Prefix_str[1])\n",
    "file_dir3 = 'submission/2019-03-01_22:25:43_fishnet150_27_submission.txt'\n",
    "score3,Prefix_str = splitscore(file_dir3)\n",
    "# \n",
    "file_dir4 = 'submission/2019-02-13_13:30:12_FeatherNet54_41_submission.txt'\n",
    "score4,Prefix_str = splitscore(file_dir4)\n",
    "# \n",
    "file_dir5 = 'submission/2019-02-13_14:13:43_fishnet150_16_submission.txt'\n",
    "score5,Prefix_str = splitscore(file_dir5)\n",
    "\n",
    "file_dir6 = 'submission/2019-02-16_19:31:04_moilenetv2_5_submission.txt'\n",
    "score6,Prefix_str = splitscore(file_dir6)\n",
    "file_dir7 = 'submission/2019-02-16_19:30:02_moilenetv2_7_submission.txt'\n",
    "score7,Prefix_str = splitscore(file_dir7)\n",
    "file_dir8 = 'submission/2019-02-16_19:28:47_moilenetv2_6_submission.txt'\n",
    "score8,Prefix_str = splitscore(file_dir8)\n",
    "\n",
    "\n",
    "file_dir9 = 'submission/2019-03-01_17:10:11_mobilelitenetB_48_submission.txt'\n",
    "score9,Prefix_str = splitscore(file_dir9)\n",
    "file_dir10 = 'submission/2019-03-01_17:38:27_mobilelitenetA_51_submission.txt'\n",
    "score10,Prefix_str = splitscore(file_dir10)\n",
    "\n",
    "# scores =[score1,score2,score3,score4,score5,score6,score7,score8,score9]\n",
    "scores = [score1,score6,score8,score5,score9,score9,score7,score2,score10,score10]\n",
    "# scores = [score1,score2,score3,score4,score5,score6,score7,score8,score9,score10]\n",
    "print(scores[0][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get real labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9608"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_ensembled_file_dir='data/val_label.txt'\n",
    "real_scores,Prefix_str = splitscore(submission_ensembled_file_dir)\n",
    "len(real_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def Average(lst): \n",
    "    return sum(lst) / len(lst) \n",
    "def fecth_ensembled_score(scores, threshold):\n",
    "    ensembled_score  = []\n",
    "    for i in range(len(score1)):\n",
    "        line_socres = [scores[j][i] for j in range(len(scores))]\n",
    "        mean_socre = Average(line_socres)\n",
    "        if mean_socre > threshold:\n",
    "            ensembled_score.append(max(line_socres))\n",
    "        else:\n",
    "            ensembled_score.append(min(line_socres))\n",
    "    return ensembled_score\n",
    "## \n",
    "def num_err(ensembled_score,threshold,real_scores):            \n",
    "    count = 0\n",
    "    for i in range(len(real_scores)):\n",
    "        if real_scores[i] == (ensembled_score[i]>0.5):\n",
    "            pass\n",
    "        else:\n",
    "            count = count + 1\n",
    "    if count < 5:\n",
    "        print('threshold: {:.3f} num_errors is {}'.format(threshold,count))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold: 0.479 num_errors is 3\n",
      "threshold: 0.480 num_errors is 3\n",
      "threshold: 0.481 num_errors is 3\n",
      "threshold: 0.482 num_errors is 3\n",
      "threshold: 0.483 num_errors is 3\n",
      "threshold: 0.484 num_errors is 3\n",
      "threshold: 0.485 num_errors is 3\n",
      "threshold: 0.486 num_errors is 3\n",
      "threshold: 0.487 num_errors is 3\n",
      "threshold: 0.488 num_errors is 3\n",
      "threshold: 0.489 num_errors is 3\n",
      "threshold: 0.490 num_errors is 3\n",
      "threshold: 0.491 num_errors is 3\n",
      "threshold: 0.492 num_errors is 3\n",
      "threshold: 0.493 num_errors is 3\n",
      "threshold: 0.494 num_errors is 3\n",
      "threshold: 0.495 num_errors is 3\n",
      "threshold: 0.496 num_errors is 3\n",
      "threshold: 0.497 num_errors is 3\n",
      "threshold: 0.498 num_errors is 3\n",
      "threshold: 0.499 num_errors is 3\n",
      "threshold: 0.500 num_errors is 3\n",
      "threshold: 0.501 num_errors is 3\n",
      "threshold: 0.502 num_errors is 3\n",
      "threshold: 0.503 num_errors is 3\n",
      "threshold: 0.504 num_errors is 3\n",
      "threshold: 0.505 num_errors is 3\n",
      "threshold: 0.506 num_errors is 3\n",
      "threshold: 0.507 num_errors is 3\n",
      "threshold: 0.508 num_errors is 3\n",
      "threshold: 0.509 num_errors is 3\n",
      "threshold: 0.510 num_errors is 3\n",
      "threshold: 0.511 num_errors is 3\n",
      "threshold: 0.512 num_errors is 3\n",
      "threshold: 0.513 num_errors is 3\n",
      "threshold: 0.514 num_errors is 3\n",
      "threshold: 0.515 num_errors is 3\n",
      "threshold: 0.516 num_errors is 3\n",
      "threshold: 0.517 num_errors is 3\n",
      "threshold: 0.518 num_errors is 3\n",
      "threshold: 0.519 num_errors is 3\n",
      "threshold: 0.520 num_errors is 3\n",
      "threshold: 0.521 num_errors is 3\n",
      "threshold: 0.522 num_errors is 3\n",
      "threshold: 0.523 num_errors is 3\n",
      "threshold: 0.524 num_errors is 3\n",
      "threshold: 0.525 num_errors is 3\n",
      "threshold: 0.526 num_errors is 3\n",
      "threshold: 0.527 num_errors is 3\n",
      "threshold: 0.528 num_errors is 3\n",
      "threshold: 0.529 num_errors is 3\n",
      "threshold: 0.530 num_errors is 3\n",
      "threshold: 0.531 num_errors is 3\n",
      "threshold: 0.532 num_errors is 3\n",
      "threshold: 0.533 num_errors is 3\n",
      "threshold: 0.534 num_errors is 3\n",
      "threshold: 0.535 num_errors is 3\n",
      "threshold: 0.536 num_errors is 3\n",
      "threshold: 0.537 num_errors is 3\n",
      "threshold: 0.538 num_errors is 2\n",
      "threshold: 0.539 num_errors is 2\n",
      "threshold: 0.540 num_errors is 2\n",
      "threshold: 0.541 num_errors is 1\n",
      "threshold: 0.542 num_errors is 1\n",
      "threshold: 0.543 num_errors is 1\n",
      "threshold: 0.544 num_errors is 1\n",
      "threshold: 0.545 num_errors is 1\n",
      "threshold: 0.546 num_errors is 1\n",
      "threshold: 0.547 num_errors is 1\n",
      "threshold: 0.548 num_errors is 1\n",
      "threshold: 0.549 num_errors is 1\n",
      "threshold: 0.550 num_errors is 1\n",
      "threshold: 0.551 num_errors is 1\n",
      "threshold: 0.552 num_errors is 1\n",
      "threshold: 0.553 num_errors is 1\n",
      "threshold: 0.554 num_errors is 1\n",
      "threshold: 0.555 num_errors is 1\n",
      "threshold: 0.556 num_errors is 1\n",
      "threshold: 0.557 num_errors is 1\n",
      "threshold: 0.558 num_errors is 1\n",
      "threshold: 0.559 num_errors is 1\n",
      "threshold: 0.560 num_errors is 1\n",
      "threshold: 0.561 num_errors is 1\n",
      "threshold: 0.562 num_errors is 1\n",
      "threshold: 0.563 num_errors is 1\n",
      "threshold: 0.564 num_errors is 1\n",
      "threshold: 0.565 num_errors is 1\n",
      "threshold: 0.566 num_errors is 1\n",
      "threshold: 0.567 num_errors is 1\n",
      "threshold: 0.568 num_errors is 1\n",
      "threshold: 0.569 num_errors is 1\n",
      "threshold: 0.570 num_errors is 1\n",
      "threshold: 0.571 num_errors is 1\n",
      "threshold: 0.572 num_errors is 2\n",
      "threshold: 0.573 num_errors is 2\n",
      "threshold: 0.574 num_errors is 2\n",
      "threshold: 0.575 num_errors is 2\n",
      "threshold: 0.576 num_errors is 2\n",
      "threshold: 0.577 num_errors is 2\n",
      "threshold: 0.578 num_errors is 2\n",
      "threshold: 0.579 num_errors is 2\n",
      "threshold: 0.580 num_errors is 2\n",
      "threshold: 0.581 num_errors is 1\n",
      "threshold: 0.582 num_errors is 1\n",
      "threshold: 0.583 num_errors is 1\n",
      "threshold: 0.584 num_errors is 1\n",
      "threshold: 0.585 num_errors is 1\n",
      "threshold: 0.586 num_errors is 1\n",
      "threshold: 0.587 num_errors is 1\n",
      "threshold: 0.588 num_errors is 1\n",
      "threshold: 0.589 num_errors is 1\n",
      "threshold: 0.590 num_errors is 1\n",
      "threshold: 0.591 num_errors is 1\n",
      "threshold: 0.592 num_errors is 1\n",
      "threshold: 0.593 num_errors is 1\n",
      "threshold: 0.594 num_errors is 1\n",
      "threshold: 0.595 num_errors is 1\n",
      "threshold: 0.596 num_errors is 1\n",
      "threshold: 0.597 num_errors is 1\n",
      "threshold: 0.598 num_errors is 1\n",
      "threshold: 0.599 num_errors is 1\n",
      "threshold: 0.600 num_errors is 1\n",
      "threshold: 0.601 num_errors is 1\n",
      "threshold: 0.602 num_errors is 1\n",
      "threshold: 0.603 num_errors is 1\n",
      "threshold: 0.604 num_errors is 1\n",
      "threshold: 0.605 num_errors is 1\n",
      "threshold: 0.606 num_errors is 1\n",
      "threshold: 0.607 num_errors is 1\n",
      "threshold: 0.608 num_errors is 1\n",
      "threshold: 0.609 num_errors is 1\n",
      "threshold: 0.610 num_errors is 1\n",
      "threshold: 0.611 num_errors is 1\n",
      "threshold: 0.612 num_errors is 1\n",
      "threshold: 0.613 num_errors is 1\n",
      "threshold: 0.614 num_errors is 1\n",
      "threshold: 0.615 num_errors is 1\n",
      "threshold: 0.616 num_errors is 1\n",
      "threshold: 0.617 num_errors is 1\n",
      "threshold: 0.618 num_errors is 1\n",
      "threshold: 0.619 num_errors is 1\n",
      "threshold: 0.620 num_errors is 1\n",
      "threshold: 0.621 num_errors is 1\n",
      "threshold: 0.622 num_errors is 1\n",
      "threshold: 0.623 num_errors is 1\n",
      "threshold: 0.624 num_errors is 1\n",
      "threshold: 0.625 num_errors is 1\n",
      "threshold: 0.626 num_errors is 1\n",
      "threshold: 0.627 num_errors is 1\n",
      "threshold: 0.628 num_errors is 1\n",
      "threshold: 0.629 num_errors is 1\n",
      "threshold: 0.630 num_errors is 1\n",
      "threshold: 0.631 num_errors is 1\n",
      "threshold: 0.632 num_errors is 1\n",
      "threshold: 0.633 num_errors is 1\n",
      "threshold: 0.634 num_errors is 1\n",
      "threshold: 0.635 num_errors is 1\n",
      "threshold: 0.636 num_errors is 1\n",
      "threshold: 0.637 num_errors is 1\n",
      "threshold: 0.638 num_errors is 1\n",
      "threshold: 0.639 num_errors is 1\n",
      "threshold: 0.640 num_errors is 1\n",
      "threshold: 0.641 num_errors is 1\n",
      "threshold: 0.642 num_errors is 1\n",
      "threshold: 0.643 num_errors is 1\n",
      "threshold: 0.644 num_errors is 1\n",
      "threshold: 0.645 num_errors is 1\n",
      "threshold: 0.646 num_errors is 1\n",
      "threshold: 0.647 num_errors is 1\n",
      "threshold: 0.648 num_errors is 1\n",
      "threshold: 0.649 num_errors is 1\n",
      "threshold: 0.650 num_errors is 1\n",
      "threshold: 0.651 num_errors is 1\n",
      "threshold: 0.652 num_errors is 1\n",
      "threshold: 0.653 num_errors is 1\n",
      "threshold: 0.654 num_errors is 1\n",
      "threshold: 0.655 num_errors is 1\n",
      "threshold: 0.656 num_errors is 1\n",
      "threshold: 0.657 num_errors is 1\n",
      "threshold: 0.658 num_errors is 1\n",
      "threshold: 0.659 num_errors is 1\n",
      "threshold: 0.660 num_errors is 1\n",
      "threshold: 0.661 num_errors is 1\n",
      "threshold: 0.662 num_errors is 1\n",
      "threshold: 0.663 num_errors is 1\n",
      "threshold: 0.664 num_errors is 1\n",
      "threshold: 0.665 num_errors is 1\n",
      "threshold: 0.666 num_errors is 1\n",
      "threshold: 0.667 num_errors is 1\n",
      "threshold: 0.668 num_errors is 1\n",
      "threshold: 0.669 num_errors is 1\n",
      "threshold: 0.670 num_errors is 1\n",
      "threshold: 0.671 num_errors is 1\n",
      "threshold: 0.672 num_errors is 1\n",
      "threshold: 0.673 num_errors is 1\n",
      "threshold: 0.674 num_errors is 1\n",
      "threshold: 0.675 num_errors is 1\n",
      "threshold: 0.676 num_errors is 1\n",
      "threshold: 0.677 num_errors is 1\n",
      "threshold: 0.678 num_errors is 1\n",
      "threshold: 0.679 num_errors is 1\n",
      "threshold: 0.680 num_errors is 1\n",
      "threshold: 0.681 num_errors is 1\n",
      "threshold: 0.682 num_errors is 1\n",
      "threshold: 0.683 num_errors is 1\n",
      "threshold: 0.684 num_errors is 1\n",
      "threshold: 0.685 num_errors is 1\n",
      "threshold: 0.686 num_errors is 1\n",
      "threshold: 0.687 num_errors is 1\n",
      "threshold: 0.688 num_errors is 1\n",
      "threshold: 0.689 num_errors is 1\n",
      "threshold: 0.690 num_errors is 1\n",
      "threshold: 0.691 num_errors is 1\n",
      "threshold: 0.692 num_errors is 1\n",
      "threshold: 0.693 num_errors is 1\n",
      "threshold: 0.694 num_errors is 1\n",
      "threshold: 0.695 num_errors is 1\n",
      "threshold: 0.696 num_errors is 1\n",
      "threshold: 0.697 num_errors is 1\n",
      "threshold: 0.698 num_errors is 1\n",
      "threshold: 0.699 num_errors is 2\n",
      "threshold: 0.700 num_errors is 2\n",
      "threshold: 0.701 num_errors is 2\n",
      "threshold: 0.702 num_errors is 2\n",
      "threshold: 0.703 num_errors is 2\n",
      "threshold: 0.704 num_errors is 2\n",
      "threshold: 0.705 num_errors is 2\n",
      "threshold: 0.706 num_errors is 2\n",
      "threshold: 0.707 num_errors is 2\n",
      "threshold: 0.708 num_errors is 2\n",
      "threshold: 0.709 num_errors is 2\n",
      "threshold: 0.710 num_errors is 2\n",
      "threshold: 0.711 num_errors is 2\n",
      "threshold: 0.712 num_errors is 2\n",
      "threshold: 0.713 num_errors is 2\n",
      "threshold: 0.714 num_errors is 2\n",
      "threshold: 0.715 num_errors is 2\n",
      "threshold: 0.716 num_errors is 2\n",
      "threshold: 0.717 num_errors is 2\n",
      "threshold: 0.718 num_errors is 2\n",
      "threshold: 0.719 num_errors is 2\n",
      "threshold: 0.720 num_errors is 4\n",
      "threshold: 0.721 num_errors is 4\n",
      "threshold: 0.722 num_errors is 4\n",
      "threshold: 0.723 num_errors is 4\n",
      "threshold: 0.724 num_errors is 4\n",
      "threshold: 0.725 num_errors is 4\n",
      "threshold: 0.726 num_errors is 4\n",
      "threshold: 0.727 num_errors is 4\n",
      "threshold: 0.728 num_errors is 4\n"
     ]
    }
   ],
   "source": [
    "for i in range(1000):\n",
    "    threshold = i / 1000\n",
    "    ensembled_score = fecth_ensembled_score(scores, threshold)\n",
    "    num_err(ensembled_score,threshold,real_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select the appropriate threshold to generate the final submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_ensembled_file_dir='submission/submission_0.53.txt'\n",
    "ensembled_file = open(submission_ensembled_file_dir,'a')\n",
    "ensembled_score = fecth_ensembled_score(scores, 0.53)\n",
    "for i in range(len(ensembled_score)):\n",
    "    ensembled_file.write(Prefix_str[i]+str(ensembled_score[i])+'\\n')\n",
    "ensembled_file.close()    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
